{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0422950",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import json\n",
    "import subprocess\n",
    "import shlex"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07b964c1",
   "metadata": {},
   "source": [
    "# Step 1: Clean cache\n",
    "\n",
    "What I did was to set a new conan cache\n",
    "\n",
    "```shell\n",
    "mkdir ~/.conan/bump_deps\n",
    "conan config set storage.path=\"./bump_deps\" \n",
    "```\n",
    "\n",
    "Then I cmake-configured a Release and a Debug build on the M1 mac (so it'll pick up most of build dependencies too since packages aren't readily available)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23839c73",
   "metadata": {},
   "source": [
    "# Supporting Classes"
   ]
  },
  {
   "cell_type": "raw",
   "id": "214273a8",
   "metadata": {},
   "source": [
    "import re\n",
    "reference_re = re.compile(r'(?P<name>[^/@]+?)/(?P<version>[^/@]+)@?(?:(?P<user>.+?)/(?P<channel>.+?))?#(?P<revision>.+)')\n",
    "reference_re.match(s).groupdict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8daf4749",
   "metadata": {},
   "outputs": [],
   "source": [
    "class PkgInfo:\n",
    "    @staticmethod\n",
    "    def from_metadata(metadata_path):\n",
    "        name, version, user, channel = p.relative_to(CONAN_CACHE).parent.parts\n",
    "\n",
    "        with open(p, \"r\") as f:\n",
    "            data = json.load(f)\n",
    "            revision = data[\"recipe\"][\"revision\"]\n",
    "\n",
    "        return PkgInfo(name=name, version=version, user=user, channel=channel, revision=revision)\n",
    "\n",
    "    @staticmethod\n",
    "    def from_str(reference):\n",
    "        n, revision = reference.split(\"#\")\n",
    "        if \"@\" in n:\n",
    "            name_version, user_channel = n.split(\"@\")\n",
    "        else:\n",
    "            name_version = n\n",
    "            user_channel = \"\"\n",
    "\n",
    "        name, version = name_version.split(\"/\")\n",
    "        if user_channel:\n",
    "            user, channel = user_channel.split(\"/\")\n",
    "        else:\n",
    "            user, channel = (None, None)\n",
    "\n",
    "        return PkgInfo(name=name, version=version, user=user, channel=channel, revision=revision)\n",
    "\n",
    "    def __init__(self, name, version, user, channel, revision):\n",
    "        self.name = name\n",
    "        self.version = version\n",
    "        self.user = None\n",
    "        if user is not None and user != \"_\":\n",
    "            self.user = user\n",
    "        self.channel = None\n",
    "        if channel is not None and channel != \"_\":\n",
    "            self.channel = channel\n",
    "\n",
    "        self.revision = revision\n",
    "\n",
    "        self.remote = \"conancenter\"\n",
    "        if self.name == \"ruby_installer\":\n",
    "            self.remote = \"bincrafters\"\n",
    "        elif self.name == \"openstudio_ruby\":\n",
    "            self.remote = \"nrel\"\n",
    "\n",
    "    def search_packages(\n",
    "        self, verbose=True, skip_shared=False, local_cache=False, arch_only=None, compiler_version_only=None\n",
    "    ):\n",
    "        \"\"\"Filters out packages (such as Windows MSVC 15)\n",
    "\n",
    "        Args:\n",
    "        ------\n",
    "\n",
    "        * skip_shared (bool): Don't keep the shared ones\n",
    "        * local_cache (bool, default False): if True, will search your cache. Otherwise will look in self.remote\n",
    "\n",
    "        * arch_only (None or str): if specified, will keep only this arch (eg: 'x86')\n",
    "        * compiler_version_only (None or str): if specified, will keep only this compiler.version (eg: '17')\n",
    "\n",
    "        Example with boost:\n",
    "        --------------------\n",
    "\n",
    "        pkg_info = PkgInfo(name='boost', version=\"1.79.0\", user=None, channel=None, revision='f664bfe40e2245fa9baf1c742591d582')\n",
    "\n",
    "        # Download everything\n",
    "        pkg_info.download_all()\n",
    "\n",
    "        !du -sh /Users/julien/.conan/bump_deps/boost/1.79.0/_/_/package/\n",
    "        21G boost/1.79.0/_/_/package/\n",
    "\n",
    "        # Filter, but keep shared=True ones\n",
    "        pkg_info.cleanup_skipped_packages(skip_shared=False)\n",
    "\n",
    "        !du -sh /Users/julien/.conan/bump_deps/boost/1.79.0/_/_/package/\n",
    "        11G boost/1.79.0/_/_/package/\n",
    "\n",
    "        # Remove the shared=True ones\n",
    "        pkg_info.cleanup_skipped_packages(skip_shared=False)\n",
    "\n",
    "        !du -sh /Users/julien/.conan/bump_deps/boost/1.79.0/_/_/package/\n",
    "        6.6G boost/1.79.0/_/_/package/\n",
    "        \"\"\"\n",
    "        json_p = Path(f\"{self.name}.json\")\n",
    "        args = [\"conan\", \"search\", \"--json\", str(json_p)]\n",
    "        if not local_cache:\n",
    "            args += [\"-r\", self.remote]\n",
    "        args += [self.reference()]\n",
    "        if verbose:\n",
    "            print(args)\n",
    "        subprocess.check_call(args, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL)\n",
    "        with open(json_p, \"r\") as f:\n",
    "            data = json.load(f)\n",
    "        json_p.unlink(missing_ok=False)  # remove tmp json\n",
    "\n",
    "        packages = data[\"results\"][0][\"items\"][0][\"packages\"]\n",
    "        keep_packages = []\n",
    "        skipped_packages = []\n",
    "        for p in packages:\n",
    "            settings = p[\"settings\"]\n",
    "            os_ = settings.get(\"os\", None)\n",
    "            compiler_version = settings.get(\"compiler.version\", None)\n",
    "            compiler = settings.get(\"compiler\", None)\n",
    "            libcxx = settings.get(\"compiler.libcxx\", None)\n",
    "\n",
    "            if arch_only is not None:\n",
    "                arch = settings.get(\"arch\", None)\n",
    "                if arch not in [arch_only, None]:\n",
    "                    if verbose:\n",
    "                        print(f\"Skipping package with arch {arch} for os {os_}, {compiler=} for pkg {self.name}\")\n",
    "                    skipped_packages.append(p)\n",
    "                    continue\n",
    "            if compiler_version_only is not None:\n",
    "                if compiler_version not in [compiler_version_only, None]:\n",
    "                    if verbose:\n",
    "                        print(\n",
    "                            f\"Skipping package with compiler.version {compiler_version} for os {os_}, {compiler=} for pkg {self.name}\"\n",
    "                        )\n",
    "                    skipped_packages.append(p)\n",
    "                    continue\n",
    "\n",
    "            is_shared = p[\"options\"].get(\"shared\", None) == \"True\"\n",
    "            if is_shared and skip_shared:\n",
    "                if verbose:\n",
    "                    print(f\"Skipping SHARED package for os {os_}, {compiler=} for pkg {self.name}\")\n",
    "                skipped_packages.append(p)\n",
    "                continue\n",
    "\n",
    "            if os_ == \"Windows\":\n",
    "                if compiler_version not in [\"16\", \"17\", None]:\n",
    "                    if verbose:\n",
    "                        print(f\"Skipping Windows {compiler_version=} for pkg {self.name}\")\n",
    "                    skipped_packages.append(p)\n",
    "                    continue\n",
    "\n",
    "                runtime = settings.get(\"compiler.runtime\", None)\n",
    "                if runtime not in [\"MD\", \"MDd\", None]:\n",
    "                    if verbose:\n",
    "                        print(f\"Skipping Windows {runtime=} for pkg {self.name}\")\n",
    "                    skipped_packages.append(p)\n",
    "                    continue\n",
    "            elif os_ == \"Linux\":\n",
    "                if compiler not in [\"gcc\", \"clang\", None]:\n",
    "                    if verbose:\n",
    "                        print(f\"Skipping Linux {compiler=} for pkg {self.name}\")\n",
    "                    skipped_packages.append(p)\n",
    "                    continue\n",
    "\n",
    "                if libcxx not in [\"libstdc++11\", \"libc++\", None]:\n",
    "                    if verbose:\n",
    "                        print(f\"Skipping Linux {libcxx=} for pkg {self.name} with ({compiler=})\")\n",
    "                    skipped_packages.append(p)\n",
    "                    continue\n",
    "\n",
    "                if compiler == \"gcc\":\n",
    "                    if compiler_version not in [\"7\", \"8\", \"9\", \"10\", \"11\", \"12\", None]:\n",
    "                        if verbose:\n",
    "                            print(f\"Skipping Linux gcc {compiler_version=} for pkg {self.name}\")\n",
    "                        skipped_packages.append(p)\n",
    "                        continue\n",
    "\n",
    "            elif os_ == \"Macos\":\n",
    "                if libcxx not in [\"libc++\", None]:\n",
    "                    if verbose:\n",
    "                        print(f\"Skipping Macos {libcxx=} for pkg {self.name} with ({compiler=})\")\n",
    "                    skipped_packages.append(p)\n",
    "                    continue\n",
    "            elif os_ is None:\n",
    "                pass\n",
    "            else:\n",
    "                print(\"Unknown os: {os_}\")\n",
    "                skipped_packages.append(p)\n",
    "                continue\n",
    "\n",
    "            keep_packages.append(p)\n",
    "\n",
    "        return keep_packages, skipped_packages\n",
    "\n",
    "    def download_all(self):\n",
    "        subprocess.check_call(\n",
    "            [\"conan\", \"download\", \"-r\", self.remote, self.reference()],\n",
    "            # stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL\n",
    "        )\n",
    "\n",
    "    def download_specific_packages(self):\n",
    "        \"\"\"Filters out the stuff we don't need by calling `search_packages`\"\"\"\n",
    "        packages, _ = self.search_packages()\n",
    "\n",
    "        for p_dict in packages:\n",
    "            print(p_dict)\n",
    "            pkg_id = p_dict[\"id\"]\n",
    "            subprocess.check_call(\n",
    "                [\"conan\", \"download\", \"-r\", self.remote, f\"{self.reference()}:{pkg_id}\"],\n",
    "                # stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL\n",
    "            )\n",
    "\n",
    "    def upload_to_nrel(self):\n",
    "        subprocess.check_call(\n",
    "            [\"conan\", \"upload\", \"-r\", \"nrel\", \"--all\", \"--parallel\", \"--no-overwrite\", \"all\", self.reference()],\n",
    "            # stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL\n",
    "        )\n",
    "\n",
    "    def upload_specific_packages_to_nrel(self, arch_only=None, compiler_version_only=None):\n",
    "        \"\"\"Filters out the stuff we don't need by calling `search_packages(local_cache=True)`\n",
    "        And upload only the packages that matches\n",
    "        \"\"\"\n",
    "        if arch_only is None and compiler_version_only is None:\n",
    "            raise ValueError(\"Provide at least one filter!\")\n",
    "        packages, _ = self.search_packages(\n",
    "            arch_only=arch_only, compiler_version_only=compiler_version_only, local_cache=True\n",
    "        )\n",
    "\n",
    "        for p_dict in packages:\n",
    "            print(p_dict)\n",
    "            pkg_id = p_dict[\"id\"]\n",
    "            args = [\n",
    "                \"conan\",\n",
    "                \"upload\",\n",
    "                \"-r\",\n",
    "                \"nrel\",\n",
    "                \"--all\",\n",
    "                \"--parallel\",\n",
    "                \"--no-overwrite\",\n",
    "                \"all\",\n",
    "                f\"{self.reference()}:{pkg_id}\",\n",
    "            ]\n",
    "            print(\" \".join(args))\n",
    "            subprocess.check_call(\n",
    "                args,\n",
    "                # stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL\n",
    "            )\n",
    "\n",
    "    def package_dir(self):\n",
    "        p = CONAN_CACHE / f\"{self.name}/{self.version}\"\n",
    "        if self.user is not None:\n",
    "            p /= f\"{self.user}/{self.channel}\"\n",
    "        else:\n",
    "            p /= \"_/_\"\n",
    "        p /= \"package\"\n",
    "        return p\n",
    "\n",
    "    def cleanup_skipped_packages(self, remote=None, skip_shared=False, arch_only=None):\n",
    "        \"\"\"if remote is none, cleans up your local cache\"\"\"\n",
    "        _, skipped_packages = self.search_packages(skip_shared=skip_shared, arch_only=arch_only)\n",
    "        for p_dict in skipped_packages:\n",
    "            pkg_id = p_dict[\"id\"]\n",
    "            cmd_args = [\"conan\", \"remove\", \"-f\", self.reference(), \"-p\", pkg_id]\n",
    "            if remote is not None:\n",
    "                cmd_args += [\"-r\", remote]\n",
    "\n",
    "            subprocess.run(cmd_args)\n",
    "\n",
    "    def reference(self):\n",
    "        s = f\"{self.name}/{self.version}@\"\n",
    "        if self.user is not None:\n",
    "            s += f\"{self.user}/{self.channel}\"\n",
    "        s += f\"#{self.revision}\"\n",
    "        return s\n",
    "\n",
    "    def __repr__(self):\n",
    "        return self.reference()\n",
    "\n",
    "    def __eq__(self, other):\n",
    "        return self.reference() == other.reference()"
   ]
  },
  {
   "cell_type": "raw",
   "id": "fbee2563",
   "metadata": {},
   "source": [
    "query_by_os = {\n",
    "    'Windows' : {'compiler.version' : [16, 17]}\n",
    "    'Macos': None,\n",
    "    'Linux':  {\n",
    "        'compiler': 'gcc',\n",
    "        'compiler.version': [7, 8, 9, 10, 11]\n",
    "    },\n",
    "}\n",
    "base_query = -q \"(os=Windows AND (compiler.version=16 OR compiler.version=17)) OR (os=Macos) OR (os=Linux AND compiler=gcc AND compiler.libcxx=libstdc++ AND (compiler.version=7 OR compiler.version=8 OR compiler.version=9 OR compiler.version=10 OR compiler.version=11))\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36278a6d",
   "metadata": {},
   "source": [
    "# Download packages"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6479c4d9",
   "metadata": {},
   "source": [
    "## Parse my local cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b45e7542",
   "metadata": {},
   "outputs": [],
   "source": [
    "CONAN_CACHE = Path(\"~/.conan/bump_deps/\").expanduser()\n",
    "\n",
    "pkg_infos = []\n",
    "for p in CONAN_CACHE.glob(\"**/metadata.json\"):\n",
    "    pkg_infos.append(PkgInfo.from_metadata(p))\n",
    "pkg_infos.sort(key=lambda p: p.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1b4bbe9",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(pkg_infos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d590aa75",
   "metadata": {},
   "outputs": [],
   "source": [
    "pkg_infos"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "462774f4",
   "metadata": {},
   "source": [
    "## Compare with last time it was run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03b8c192",
   "metadata": {},
   "outputs": [],
   "source": [
    "old_refs = [\n",
    "    \"b2/4.8.0@#012527a73298c09865ac86e6921b8bc9\",\n",
    "    \"benchmark/1.6.1@#94c40ebf065e3b20cab6a4f1b03a65fe\",\n",
    "    \"bison/3.7.6@#de3449489624dbc45cfb8a868818def8\",\n",
    "    \"boost/1.79.0@#f664bfe40e2245fa9baf1c742591d582\",\n",
    "    \"bzip2/1.0.8@#b056f852bd2d5af96fc7171aadfd6c0b\",\n",
    "    \"cpprestsdk/2.10.18@#df2f6ac88e47cadd9c9e8e0971e00d89\",\n",
    "    \"flex/2.6.4@#e4696e6333f2d486e7073de6a9559712\",\n",
    "    \"fmt/8.1.1@#b3e969f8561a85087bd0365c09bbf4fb\",\n",
    "    \"gdbm/1.19@#c420bc00f3cc629aef665fdc3100f926\",\n",
    "    \"geographiclib/1.52@#76536a9315a003ef3511919310b2fe37\",\n",
    "    \"gmp/6.2.1@#2011237c81178d014a4c08ae40cfe0cb\",\n",
    "    \"gnu-config/cci.20210814@#58573fa18a083c1ccb883b392b826bb2\",\n",
    "    \"gtest/1.11.0@#8aca975046f1b60c660ee9d066764d69\",\n",
    "    \"jsoncpp/1.9.5@#536d080aa154e5853332339bf576747c\",\n",
    "    \"libbacktrace/cci.20210118@#b707cfa5d717e9e1401017c741ad8f6c\",\n",
    "    \"libffi/3.4.2@#4121e32bfd908d32864cb97643b2b5a9\",\n",
    "    \"libiconv/1.16@#eae489614aa6b1b8ca652cc33d3c26a9\",\n",
    "    \"libyaml/0.2.5@#5bdf6971928f655a994646f8dbb221e4\",\n",
    "    \"m4/1.4.19@#d9741f0aa0ac45e6b54a59f79e32ac81\",\n",
    "    \"minizip/1.2.12@#0b5296887a2558500d0323c6c94c8d02\",\n",
    "    \"nlohmann_json/3.9.1@#304649bcd7dae8fa24a2356e9437f9ad\",\n",
    "    \"openssl/1.1.1o@#213dbdeb846a4b40b4dec36cf2e673d7\",\n",
    "    \"openstudio_ruby/2.7.2@nrel/testing#98444b7bc8d391ea1521d7f79d4d4926\",\n",
    "    \"pcre/8.45@#9158a180422a0d4dc01c668cbee37100\",\n",
    "    \"pugixml/1.12.1@#5a39f82651eba3e7d6197903a3202e21\",\n",
    "    \"readline/8.1.2@#ae31d1d71b027b0fe35903eb6c2e8333\",\n",
    "    \"ruby_installer/2.7.3@bincrafters/stable#90fad7a169f6cb267c3e2e6aee106566\",\n",
    "    \"sqlite3/3.38.5@#010911927ce1889b5cf824f45e7cd3d2\",\n",
    "    \"stb/20200203@#cba8fa43f7a0ea5389896744664823c9\",\n",
    "    \"swig/4.0.2@#9fcccb1e39eed9acd53a4363d8129be5\",\n",
    "    \"termcap/1.3.1@#733491d29bb22e81d06edaf78c6727c9\",\n",
    "    \"tinygltf/2.5.0@#c8b2aca9505e86312bb42aa0e1c639ec\",\n",
    "    \"websocketpp/0.8.2@#3fd704c4c5388d9c08b11af86f79f616\",\n",
    "    \"zlib/1.2.12@#3b9e037ae1c615d045a06c67d88491ae\",\n",
    "]\n",
    "old_pkgs = [PkgInfo.from_str(x) for x in old_refs]"
   ]
  },
  {
   "cell_type": "raw",
   "id": "6c54343c",
   "metadata": {},
   "source": [
    "# I did Unix first, then windows\n",
    "extra_win_build_requires = set(x.reference() for x in pkg_infos) - set(x.reference() for x in old_pkgs)\n",
    "extra_win_build_requires\n",
    "set(x.name for x in pkg_infos) - set(x.name for x in old_pkgs)\n",
    "\n",
    "extras_unix = set(x.reference() for x in old_pkgs) - set(x.reference() for x in pkg_infos)\n",
    "extras_unix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "748d7396",
   "metadata": {},
   "source": [
    "strawberryperl is a build_requires of openssl\n",
    "nasm is a build_requires of openssl\n",
    "winflexbison is a build_requires of swig\n",
    "autoconf is a requires of automake\n",
    "automake is a build_requires of bison (which is always a build_requires, and one of swig)\n",
    "msys2 is always a build_requires (of automake, and of swig)\n",
    "\n",
    "=> if we prebuild swig and openssl, these shouldn't be needed"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1dca2257",
   "metadata": {},
   "source": [
    "### test bed a single package (boost)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "dcf949e7",
   "metadata": {},
   "source": [
    "pkg_info = [x for x in pkg_infos if x.name == 'boost'][0]\n",
    "keep_packages, skipped_packages = pkg_info.search_packages(verbose=True, skip_shared=True)\n",
    "len(keep_packages), len(skipped_packages)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03cbb678",
   "metadata": {},
   "source": [
    "## Compare packages I have in my cache with the conan.lock produced when building OS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "441999e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "conan_lock = \"/Users/julien/Software/Others/OS-build-bump/conan.lock\"\n",
    "with open(conan_lock, \"r\") as f:\n",
    "    conan_lock_data = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "451b3b94",
   "metadata": {},
   "outputs": [],
   "source": [
    "conan_lock_data.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eefde698",
   "metadata": {},
   "outputs": [],
   "source": [
    "pkg_infos_lock = []\n",
    "for k, node in conan_lock_data[\"graph_lock\"][\"nodes\"].items():\n",
    "    if not \"ref\" in node:\n",
    "        print(f\"{k=} has no ref (it's node 0, that's normal)\")\n",
    "        continue\n",
    "    pkg_infos_lock.append(PkgInfo.from_str(reference=node[\"ref\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0980364f",
   "metadata": {},
   "outputs": [],
   "source": [
    "pkg_infos_lock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62111695",
   "metadata": {},
   "outputs": [],
   "source": [
    "set(x.reference() for x in pkg_infos) - set(x.reference() for x in pkg_infos_lock)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5474c1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "set(x.reference() for x in pkg_infos_lock) - set(x.reference() for x in pkg_infos)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8859fe1f",
   "metadata": {},
   "source": [
    "## Download all packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7865dd1a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for pkg_info in pkg_infos:\n",
    "    if pkg_info.name == \"openstudio_ruby\":\n",
    "        continue\n",
    "\n",
    "    print(pkg_info.name)\n",
    "\n",
    "    # Filter before downloading:\n",
    "    # keep_packages, skip_packages = pkg_info.search_packages()\n",
    "    # pkg_info.download_specific_packages()\n",
    "\n",
    "    # download_all has the benefit of running in parallel... so it's faster provided you have a good connection\n",
    "    # We'll clean it up later\n",
    "    pkg_info.download_all()\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "897629af",
   "metadata": {},
   "source": [
    "# Clean up unwanted binaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6bc1e16",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for pkg_info in pkg_infos:\n",
    "    if pkg_info.name == \"openstudio_ruby\":\n",
    "        continue\n",
    "\n",
    "    print(pkg_info.name)\n",
    "    pkg_info.cleanup_skipped_packages(skip_shared=True)\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee215576",
   "metadata": {},
   "source": [
    "# Upload to NREL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41fe558f",
   "metadata": {},
   "outputs": [],
   "source": [
    "for pkg_info in pkg_infos:\n",
    "    if pkg_info.name == \"openstudio_ruby\":\n",
    "        continue\n",
    "    print(pkg_info.name)\n",
    "    pkg_info.upload_to_nrel()\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ece9382e",
   "metadata": {},
   "source": [
    "# In one go"
   ]
  },
  {
   "cell_type": "raw",
   "id": "c3a03a4d",
   "metadata": {},
   "source": [
    "pkg_infos = []\n",
    "for p in CONAN_CACHE.glob('**/metadata.json'):\n",
    "    pkg_infos.append(PkgInfo.from_metadata(p))\n",
    "pkg_infos.sort(key=lambda p: p.name)\n",
    "\n",
    "for pkg_info in pkg_infos:\n",
    "    if pkg_info.name == 'openstudio_ruby':\n",
    "        continue\n",
    "    print(pkg_info.name)\n",
    "    pkg_info.download_all()\n",
    "    pkg_info.cleanup_skipped_packages(skip_shared=True)\n",
    "    pkg_info.upload_to_nrel()\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4fdde717",
   "metadata": {},
   "source": [
    "# Windows"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "73e71242",
   "metadata": {},
   "source": [
    "## Upload unusual configurations: the win32 and MSVC 17 stuff\n",
    "\n",
    "That's not part of conancenter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4d0ca0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Not sure whether I should upload those or not...?\n",
    "extra_win_build_requires = [\"autoconf\", \"automake\", \"msys2\", \"nasm\", \"strawberryperl\", \"winflexbison\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f1ac5df",
   "metadata": {},
   "outputs": [],
   "source": [
    "pkg_infos_regular = [x for x in pkg_infos if not x.name in extra_win_build_requires]\n",
    "len(pkg_infos_regular)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce6109ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "pkg_infos_regular"
   ]
  },
  {
   "cell_type": "raw",
   "id": "030861e2",
   "metadata": {},
   "source": [
    "pkg_info = next(p for p in pkg_infos if p.name == 'boost')\n",
    "keep_packages, skipped_packages = pkg_info.search_packages(verbose=False, compiler_version_only='17', local_cache=True)\n",
    "for p_dict in keep_packages:\n",
    "    pkg_id = p_dict['id']\n",
    "    args = ['conan', 'upload', '-r', 'nrel', '--all', '--parallel',\n",
    "     '--no-overwrite', 'all', f\"{pkg_info.reference()}:{pkg_id}\"]\n",
    "    print(\" \".join(args))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "990817b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "for pkg_info in pkg_infos_regular:\n",
    "    keep_packages, _ = pkg_info.search_packages(\n",
    "        verbose=False,\n",
    "        local_cache=True,\n",
    "        compiler_version_only=\"17\",\n",
    "        # arch_only='x86'\n",
    "    )\n",
    "    if not keep_packages:\n",
    "        print(pkg_info)\n",
    "    else:\n",
    "        print(f\"Found {len(keep_packages)} for {pkg_info}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ef043f2",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for pkg_info in pkg_infos_regular:\n",
    "    # if pkg_info.name == 'boost':\n",
    "    #    continue\n",
    "    pkg_info.upload_specific_packages_to_nrel(\n",
    "        # arch_only='x86',\n",
    "        compiler_version_only=\"17\"\n",
    "    )\n",
    "    done.append(pkg_info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c055c62",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"DONE\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04ccabf1",
   "metadata": {},
   "source": [
    "## Upload the windows specific build_requires"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1dc28167",
   "metadata": {},
   "outputs": [],
   "source": [
    "extra_win_build_requires"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "418701c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "pkg_infos_win_build_requires = [x for x in pkg_infos if x.name in extra_win_build_requires]\n",
    "pkg_infos_win_build_requires"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cacebfce",
   "metadata": {},
   "outputs": [],
   "source": [
    "for pkg_info in pkg_infos_win_build_requires:\n",
    "    print(pkg_info.name)\n",
    "    pkg_info.download_all()\n",
    "    pkg_info.cleanup_skipped_packages(skip_shared=True)\n",
    "    pkg_info.upload_to_nrel()\n",
    "    print(\"\\n\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "345.6px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
